# with open('train.txt',encoding="utf-8") as reader, open('2.txt', 'w',encoding="utf-8") as writer:
#     for index, line in enumerate(reader):
#         if index > 170143:
#             writer.write(line)



# ##按标点分行
# list_biaodian=['。','！','？','，','：','；']
#
# with open('test1.txt', 'r', encoding="gbk") as f:
#     for line in f:
#         content = line.split(' ')[0]
#         tag = line.split(" ")[1].split("\n")[0]
#         if tag=='B':
#             tag='B-LOC'
#         elif tag=='I':
#             tag = 'I-LOC'
#         lines=content+' '+tag+'\n'
#         if content in list_biaodian:
#             f1 = open('test.txt', 'a', encoding="utf-8")
#             f1.write(lines)
#             f1.write('\n')
#             f1.close()
#         else:
#             f1 = open('test.txt', 'a', encoding="utf-8")
#             f1.write(lines)
#             f1.close()

# 去除空行
# with open('test1.txt', 'r', encoding="utf-8") as f:
#     for line in f:
#         if line != '\n':
#             f1 = open('test_1.txt', 'a', encoding="utf-8")
#             f1.write(line)
#             f1.close()
#


#处理excel
# with open('章节.txt', 'r', encoding="GBK") as f:
#     for line in f:
#         if line == '\t\n':
#             f1 = open('章节_1.txt', 'a', encoding="utf-8")
#             f1.write('\n')
#             f1.close()
#         else:
#             word=line.split("\t")[0]
#             label=line.split("\t")[2].split("\n")[0]
#             f1 = open('章节_1.txt', 'a', encoding="utf-8")
#             f1.write(word)
#             f1.write(' ')
#             f1.write(label)
#             f1.write('\n')
#             f1.close()
#
#
# #构建词典
# dict1={}
# text=''
# with open('章节_1.txt', 'r', encoding="utf-8") as f:
#      for line in f:
#         if line != '\n':
#             content = line.split(' ')[0]
#             tag = line.split(" ")[1].split("\n")[0]
#             if tag == 'O':
#                 if text != '':
#                     dict1.setdefault(text)
#                     if dict1[text] != None:
#                         dict1[text]=dict1[text]+1
#                     else:
#                         dict1[text] = 1
#                     text = ''
#             else :
#                 text=text+content
#
# dict_sort_list = sorted(dict1.items(), key=lambda x: x[1])
# dict_sort_list.reverse()
# dict_sort=dict(dict_sort_list)
# print(len(dict_sort))
#
#
# for key in dict_sort:
#     if len(key)>1:
#         f1 = open('../dict/第十章关键词.txt', 'a', encoding="utf-8")
#         f1.write(key)
#         f1.write(' ')
#         f1.write(str(dict_sort[key]))
#         f1.write('\n')
#         f1.close()





# dict2={}
# text=''
# with open('../output/label_test.txt', 'r', encoding="utf-8") as f:
#      for line in f:
#         if line != '\n':
#             content = line.split(' ')[0]
#             tag = line.split(" ")[2].split("\n")[0]
#             if tag == 'O':
#                 if text != '':
#                     dict2.setdefault(text)
#                     text = ''
#             else :
#                 text=text+content
#
#



#
# dict_sort_list_2 = sorted(dict2.items(), key=lambda x: x[0])
# dict_sort_2=dict(dict_sort_list_2)
# print(len(dict_sort_2))
#
# list_1=[]
# for key in dict_sort:
#     if key not in dict_sort_2:
#         list_1.append(key)
# list_2=[]
# for key in dict_sort_2:
#     if key not in dict_sort:
#         list_2.append(key)
# print("预测时未识别出的实体：",list_1)
# print("预测时添加的实体",list_2)





#预测标签
# with open('train.txt', 'r', encoding="GBK") as f:
#      for line in f:
#          if line==' \n':
#              f1 = open('train_label.txt', 'a', encoding="utf-8")
#              f1.write('\n')
#              f1.close()
#          else:
#              f1 = open('train_label.txt', 'a', encoding="utf-8")
#              f1.write(line)
#              f1.close()



#词典合并

# dict1={}
# with open('分词.txt', 'r', encoding="gbk") as f:
#      for line in f:
#         content = line.split('\t')[0]
#         cishu = int(line.split("\t")[1].split("\n")[0])
#         if content in dict1:
#             dict1[content]=dict1[content]+cishu
#         else:
#             dict1[content]=cishu
# #print(dict1)
# dict_sort_list = sorted(dict1.items(), key=lambda x: x[1])
# dict_sort_list.reverse()
# dict_sort=dict(dict_sort_list)
# print(len(dict_sort))
#
#
# for key in dict_sort:
#     if len(key)>1:
#         f1 = open('../dict/章节关键词汇总.txt', 'a', encoding="utf-8")
#         f1.write(key)
#         f1.write(' ')
#         f1.write(str(dict_sort[key]))
#         f1.write('\n')
#         f1.close()

#处理excel转换后txt
# with open('test_1.txt', 'r', encoding="gbk") as f:
#     for line in f:
#             if line != ' \n':
#                 f1 = open('test.txt', 'a', encoding="utf-8")
#                 f1.write(line)
#                 f1.close()
#             else:
#                 f1 = open('test.txt', 'a', encoding="utf-8")
#                 f1.write('\n')
#                 f1.close()



# #输出测试文件中的词汇变化
# def list_deduplication(list):
#     list0=[]
#     for i in list:
#         if i not in list0:
#             list0.append(i)
#     return list0
#
# text=''
# list2=[]
# with open('output_tongji/label_test_5.txt', 'r', encoding="utf-8") as f:
#     for line in f:
#         if line != '\n':
#             content = line.split(' ')[0]
#             tag = line.split(" ")[2].split("\n")[0]
#             if tag == 'O':
#                 if text != '':
#                     list2.append(text)
#                     text = ''
#             else :
#                 text=text+content
#
# print(len(list2))
# print(len(list_deduplication(list2)))




